#!/usr/bin/python
# -*- coding: UTF-8 -*-
import scrapy
from moviefetch.items import MovieItem
import re
# import etree
class MovieSpider(scrapy.Spider):
    name = 'movie'
    allowed_domains = ["www.dy2018.com"]
    start_urls = [
                "https://www.dy2018.com/",
                "https://www.dy2018.com/html/gndy/dyzz/index.html"
                ]
   
    common_url = "https://www.dy2018.com"

    def request_url(self,url):
        if url.startswith(self.common_url):
            return url
        return self.common_url + url

    def parse(self,response):

        current_url = response.url

        for i in response.xpath("//a"):
            href = i.xpath("@href").extract()[0]

            if href is not None:
                href = href.encode('utf-8')
                if re.match('/i/\d+.html',href):
                    yield scrapy.Request(self.request_url(href),self.parse)
                    continue

                if re.match('/\d+/',href):
                    yield scrapy.Request(self.request_url(href),self.parse)
                    continue

                if re.match('/html/(.)+[index.html|index_\d+.html]?',href):
                    # print(href)
                    yield scrapy.Request(self.request_url(href),self.parse)
                    continue
                
        # next_page_urls = response.xpath("//div[@class='x']/a/@href").extract()
        # if len(next_page_urls) > 0:
        #     yield scrapy.Request(self.common_url + str(next_page_urls[0]),self.parse)

        #https://www.dy2018.com/html/gndy/dyzz/20081111/15326.html
        


        if re.match("(https://www.dy2018.com)?/i/\d+.html",current_url) or re.match("(https://www.dy2018.com)?/(html/(.)+/\d+.html)",current_url):
            title =  response.xpath("/html/head/title/text()").extract()
            item = MovieItem()
            item["url"] = current_url
            item["title"] = title[0]


            nums = re.findall('/[1-9]\d*.html',current_url)
            if len(nums):
                number = nums[0]
                if len(number) > 6 :
                    number = number[1:]
                item["code"] = number
            yield item